package cnki.fetch;

import cnki.db.sql.Operator;
import cnki.entity.ArticleAbstractInfoEntity;
import com.han.http.client.exception.NullContentMatchedException;
import com.han.http.util.Regex;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.*;

import static com.han.http.util.Print.printlnErr;
import static com.han.http.util.Print.printlnInfo;

/**
 * @author: Han
 * @date: 2018/8/29 9:53
 * @dscription: 获取知网指定期刊文章列表和内容
 */
public class ArticleUrlFetcher {

    private Operator operator;

    private ArticleUrlFetcher() {
        operator = Operator.getInstance();
    }

    /**
     * 获取计算机学报文章列表
     */
    private void getJSJXArticleList() {
        Map<String, TreeSet<String>> issueMap = new TreeMap<>();
        issueMap.putAll(fillIssueMap(1987, 2018, 1, 12));
        issueMap.putAll(fillIssueMap(1985, 1986, 1, 6));
        issueMap.putAll(fillIssueMap(1979, 1980, 1, 4));
        issueMap.putAll(fillIssueMap(1978, 1978, 1, 2));
        getAndStore(issueMap, "JSJX", "计算机学报");
    }

    /**
     * 获取计算机工程文章列表
     */
    private void getJSJCArticleList() {
        Map<String, TreeSet<String>> issueMap = new TreeMap<>();
        issueMap.putAll(fillIssueMap(2018, 2018, 1, 8));
        issueMap.putAll(fillIssueMap(2013, 2017, 1, 12));
        issueMap.putAll(fillIssueMap(2012, 2012, 1, 24));
        issueMap.putAll(fillIssueMap(2011, 2011, 1, 24, "S1"));
        issueMap.putAll(fillIssueMap(2009, 2010, 1, 24));
        issueMap.putAll(fillIssueMap(2008, 2008, 1, 24, "S1"));
        issueMap.putAll(fillIssueMap(2006, 2007, 1, 24));
        issueMap.putAll(fillIssueMap(2004, 2005, 1, 24, "S1"));
        issueMap.putAll(fillIssueMap(2003, 2003, 1, 22));
        issueMap.putAll(fillIssueMap(2002, 2002, 1, 12, "S1"));
        issueMap.putAll(fillIssueMap(2001, 2001, 1, 12));
        issueMap.putAll(fillIssueMap(1999, 2000, 1, 12, "S1"));
        issueMap.putAll(fillIssueMap(1998, 1998, 1, 12));
        issueMap.putAll(fillIssueMap(1994, 1997, 1, 6, "S1"));
        issueMap.putAll(fillIssueMap(1984, 1993, 1, 6));
        issueMap.putAll(fillIssueMap(1980, 1983, 1, 4));
        issueMap.putAll(fillIssueMap(1979, 1979, 1, 3));
        issueMap.putAll(fillIssueMap(1978, 1978, 1, 4));
        issueMap.put("1977", new TreeSet<>(Arrays.asList("Z1", "04", "01")));
        issueMap.put("1976", new TreeSet<>(Arrays.asList("Z1", "01")));
        issueMap.putAll(fillIssueMap(1975, 1975, 1, 2));
        getAndStore(issueMap, "JSJC", "计算机工程");
    }

    /**
     * 获取公共管理学报文章列表
     */
    private void getGGGLArticleList() {
        Map<String, TreeSet<String>> issueMap = new TreeMap<>();
        issueMap.putAll(fillIssueMap(2018, 2018, 1, 3));
        issueMap.putAll(fillIssueMap(2004, 2017, 1, 4));
        getAndStore(issueMap, "GGGL", "公共管理学报");
    }

    /**
     * 获取系统工程文章列表
     */
    private void getGCXTArticleList() {
        Map<String, TreeSet<String>> issueMap = new TreeMap<>();
        issueMap.putAll(fillIssueMap(2018, 2018, 1, 2));
        issueMap.putAll(fillIssueMap(2004, 2017, 1, 12));
        issueMap.putAll(fillIssueMap(1986, 2003, 1, 6));
        issueMap.putAll(fillIssueMap(1984, 1985, 1, 4));
        issueMap.putAll(fillIssueMap(1983, 1983, 1, 2));
        getAndStore(issueMap, "GCXT", "系统工程");
    }

    /**
     * 获取清华大学学报(自然科学)文章列表
     */
    private void getQHXBArticleList() {
        Map<String, TreeSet<String>> issueMap = new TreeMap<>();
        issueMap.putAll(fillIssueMap(2018, 2018, 1, 7));
        issueMap.putAll(fillIssueMap(2012, 2017, 1, 12));
        issueMap.putAll(fillIssueMap(2011, 2011, 1, 12, "S1"));
        issueMap.putAll(fillIssueMap(2010, 2010, 1, 12));
        issueMap.putAll(fillIssueMap(2007, 2009, 1, 12, "S1", "S2"));
        issueMap.putAll(fillIssueMap(2005, 2006, 1, 12, "S1"));
        issueMap.putAll(fillIssueMap(2003, 2004, 1, 12));
        issueMap.putAll(fillIssueMap(2002, 2002, 1, 12, "S1"));
        issueMap.putAll(fillIssueMap(2001, 2001, 1, 12, "Z1"));
        issueMap.putAll(fillIssueMap(1997, 2000, 1, 12, "S1"));
        issueMap.putAll(fillIssueMap(1996, 1996, 1, 12));
        issueMap.putAll(fillIssueMap(1986, 1995, 1, 6));
        issueMap.putAll(fillIssueMap(1978, 1985, 1, 4));
        issueMap.putAll(fillIssueMap(1976, 1977, 1, 2));
        issueMap.putAll(fillIssueMap(1975, 1975, 1, 4));
        issueMap.putAll(fillIssueMap(1974, 1974, 1, 2, "S1", "S2"));
        issueMap.putAll(fillIssueMap(1973, 1973, 1, 4));
        issueMap.putAll(fillIssueMap(1964, 1965, 1, 4));
        issueMap.putAll(fillIssueMap(1963, 1963, 1, 6));
        issueMap.putAll(fillIssueMap(1962, 1962, 0, 0));
        issueMap.putAll(fillIssueMap(1960, 1961, 1, 2));
        issueMap.putAll(fillIssueMap(1959, 1959, 1, 5));
        issueMap.putAll(fillIssueMap(1958, 1958, 1, 3, "S1", "S2", "S3"));
        issueMap.putAll(fillIssueMap(1957, 1957, 1, 2));
        issueMap.putAll(fillIssueMap(1955, 1956, 0, 0));
        issueMap.putAll(fillIssueMap(1950, 1950, 1, -1, "S1"));
        issueMap.putAll(fillIssueMap(1948, 1948, 1, 2, "S1", "S2"));
        issueMap.putAll(fillIssueMap(1947, 1947, 0, 0, "S1"));
        issueMap.putAll(fillIssueMap(1941, 1941, 0, 0));
        issueMap.putAll(fillIssueMap(1937, 1937, 1, 3));
        issueMap.putAll(fillIssueMap(1934, 1936, 1, 4));
        issueMap.putAll(fillIssueMap(1933, 1933, 0, 0));
        issueMap.putAll(fillIssueMap(1932, 1932, 1, 2, "S1"));
        issueMap.putAll(fillIssueMap(1931, 1931, 0, 0, "S1"));
        issueMap.putAll(fillIssueMap(1930, 1930, 1, 2));
        issueMap.putAll(fillIssueMap(1928, 1928, 0, 0));
        issueMap.putAll(fillIssueMap(1924, 1927, 1, 2));
        issueMap.putAll(fillIssueMap(1918, 1918, 0, 0));
        issueMap.putAll(fillIssueMap(1917, 1917, 1, 3));
        issueMap.putAll(fillIssueMap(1915, 1916, 0, 0));
        getAndStore(issueMap, "QHXB", "清华大学学报");
    }

    /**
     * 获取计算机科学文章列表
     */
    private void getJSJAArticleList() {
        Map<String, TreeSet<String>> issueMap = new TreeMap<>();
        issueMap.putAll(fillIssueMap(2018, 2018, 1, 8, "S1"));
        issueMap.putAll(fillIssueMap(2013, 2017, 1, 12, "S1", "S2"));
        issueMap.putAll(fillIssueMap(2012, 2012, 1, 12, "S1", "S2", "S3"));
        issueMap.putAll(fillIssueMap(2011, 2011, 1, 12, "S1"));
        issueMap.putAll(fillIssueMap(1999, 2010, 1, 12));
        issueMap.putAll(fillIssueMap(1991, 1998, 1, 6));
        issueMap.putAll(fillIssueMap(1990, 1990, 1, 5));
        issueMap.putAll(fillIssueMap(1979, 1989, 1, 6));
        getAndStore(issueMap, "JSJA", "计算机科学");
    }

    /**
     * 获取科学学研究文章列表
     */
    private void getKXYJArticleList() {
        Map<String, TreeSet<String>> issueMap = new TreeMap<>();
        issueMap.putAll(fillIssueMap(2018, 2018, 1, 8));
        issueMap.putAll(fillIssueMap(2009, 2017, 1, 12));
        issueMap.putAll(fillIssueMap(2006, 2008, 1, 6, "S1", "S2"));
        issueMap.putAll(fillIssueMap(2003, 2005, 1, 6, "S1"));
        issueMap.putAll(fillIssueMap(2002, 2002, 1, 6));
        issueMap.putAll(fillIssueMap(1984, 2001, 1, 4));
        issueMap.putAll(fillIssueMap(1983, 1983, 1, 2));
        getAndStore(issueMap, "KXYJ", "科学学研究");
    }

    /**
     * 获取管理学报文章列表
     */
    private void getGLXBArticleList() {
        Map<String, TreeSet<String>> issueMap = new TreeMap<>();
        issueMap.putAll(fillIssueMap(2018, 2018, 1, 9));
        issueMap.putAll(fillIssueMap(2009, 2017, 1, 12));
        issueMap.putAll(fillIssueMap(2006, 2008, 1, 6));
        issueMap.putAll(fillIssueMap(2005, 2005, 1, 6, "S2"));
        issueMap.putAll(fillIssueMap(2004, 2004, 1, 3));
        getAndStore(issueMap, "GLXB", "管理学报");
    }

    /**
     * 使用指定年月定义刊行日期
     *
     * @param startYear     开始年
     * @param endYear       结束年
     * @param startIssue    开始月
     * @param endIssue      结束月
     * @param fragmentIssue 不在startIssue->endIssue之间的月刊号
     * @return 填充完毕的刊号map
     */
    private Map<String, TreeSet<String>> fillIssueMap(int startYear, int endYear, int startIssue, int endIssue,
                                                      String... fragmentIssue) {
        Map<String, TreeSet<String>> issueMap = new HashMap<>(10);
        for (int i = startYear; i <= endYear; i++) {
            TreeSet<String> issueList = new TreeSet<>();
            for (int j = startIssue; j <= endIssue; j++) {
                String issue = j + "";
                if (j <= 9) {
                    issue = "0" + j;
                }
                issueList.add(issue);
            }
            List<String> fragmentList = Arrays.asList(fragmentIssue);
            issueList.addAll(fragmentList);
            issueMap.put(i + "", issueList);
        }
        return issueMap;
    }

    /**
     * 下载文章链接并存储
     *
     * @param issueMap    刊号
     * @param journalName 刊名简称
     * @param source      来源
     */
    private void getAndStore(Map<String, TreeSet<String>> issueMap, String journalName, String source) {
        Set<String> yearSet = issueMap.keySet();
        int nullUrlCount = 0;
        //如果连续三次下载不到url,中断
        for (String year : yearSet) {
            TreeSet<String> issueList = issueMap.get(year);
            for (String issue : issueList) {
                List<String> detailList = getBasicCatalog(year, issue, journalName);
                detailList = washDetailList(detailList);
                if (detailList.isEmpty()) {
                    nullUrlCount++;
                } else {
                    nullUrlCount = 0;
                }
                if (nullUrlCount >= 3) {
                    try {
                        Thread.sleep(1000 * 60 * 30);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
                printlnInfo(source + ":" + year + "-" + issue + " " + detailList);
                for (String url : detailList) {
                    ArticleAbstractInfoEntity entity = new ArticleAbstractInfoEntity(url, "", "", "", "", "", "",
                            "", "", year + "-" + issue, "", "");
                    operator.insertAbstractInfo(entity);
                }
            }
        }
    }

    /**
     * 清洗url
     *
     * @param list 原始url
     * @return 清洗后的url
     */
    private List<String> washDetailList(List<String> list) {
        List<String> newList = new ArrayList<>();
        for (String u : list) {
            try {
                String dbCode = Regex.matchString(u, "(?<=dbCode=)((?!&).)*");
                String fileName = Regex.matchString(u, "(?<=filename=)((?!&).)*");
                String dbName = Regex.matchString(u, "(?<=tableName=)((?!&).)*");
                String d = String.format("http://kns.cnki.net/kcms/detail/detail.aspx?dbcode=%s&filename=%s&dbname=%s", dbCode, fileName, dbName);
                newList.add(d);
            } catch (NullContentMatchedException e) {
                e.printStackTrace();
            }
        }
        return newList;
    }

    /**
     * 获取期刊文章列表并解析出url
     *
     * @param year        期刊发表年份
     * @param issue       期刊发表刊号
     * @param journalName 期刊名缩写
     * @return 期刊文章url列表
     */
    private List<String> getBasicCatalog(String year, String issue, String journalName) {
        List<String> detailList = new ArrayList<>();

        Map<String, String> header = new HashMap<>(5);
        header.put("Host", "navi.cnki.net");
        header.put("Connection", "keep-alive");
        header.put("Content-Length", "37");
        header.put("Accept", "*/*");
        header.put("Origin", "http://navi.cnki.net");
        header.put("X-Requested-With", "XMLHttpRequest");
        header.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
                "(KHTML, like Gecko) Chrome/70.0.3528.4 Safari/537.36");
        header.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
        header.put("Referer", "http://navi.cnki.net/KNavi/JournalDetail?pcode=CJFD&pykm=" + journalName);
        header.put("Accept-Encoding", "gzip, deflate");
        header.put("Accept-Language", "en,zh-CN;q=0.9,zh;q=0.8");
        header.put("Cookie", "ASP.NET_SessionId=sodckmf4ufks4lsaijsulm2i; SID_navi=1201610; cnkiUserKey=f4cd35b6-8b9d-c5ce-b16e-c73fdde0b054; Ecp_ClientId=4180903102301191823; UM_distinctid=1659d744d1f18f-02dd64fd07d217-6154702a-1fa400-1659d744d208a1; Ecp_IpLoginFail=18090360.174.195.245; LID=WEEvREcwSlJHSldTTEYzVDhsOWprSUl1UFlPNUI1MmlaWFpEUVNMUTNnVT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!; Ecp_session=1");

        Map<String, String> data = new HashMap<>(5);
        data.put("year", year + "");
        data.put("issue", issue);
        data.put("pykm", journalName);
        data.put("pageIdx", "0");
        data.put("pcode", "CJFD");
        String url = String.format("http://navi.cnki" +
                        ".net/knavi/JournalDetail/GetArticleList?year=%s&issue=%s&pykm=%s&pageIdx=0&pcode=CJFD"
                , year, issue, journalName);
        //连接对象
        Connection connection = Jsoup.connect(url);
        //请求头
        connection.headers(header);
        //post数据
        connection.data(data);
        //超时时间
        connection.timeout(1000 * 30);
        Random random = new Random();
        try {
            Document document = connection.post();
            //抽取元素集
            Elements catalogEle = document.getElementsByClass("name");
            //遍历元素集,获取里面的链接
            for (Element titleEle : catalogEle) {
                try {
                    Element tempEle = titleEle.select("a").first();
                    if (Objects.isNull(tempEle)) {
                        continue;
                    }
                    String href = tempEle.attr("href");
                    detailList.add(href);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        } catch (IOException e) {
            printlnErr(journalName + ":" + year + "-" + issue + "抽取失败");
            e.printStackTrace();
        } finally {
            try {
                int interval = random.nextInt(1000 * 27) + 1000 * 5;
                printlnInfo("休眠" + interval + "毫秒");
                Thread.sleep(interval);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }

        return detailList;
    }

}
